This notebook contains plots for the Apple Watch Movelet project comparing the effect of setting different run-time parameter combinations (i.e. number of PCs, including both hands, dominant hand only, or nondominant hand only, number of med-taking sessions to include in the training set) on various prediction metrics.

library(dplyr)
library(tidyr)
library(ggplot2)
library(ggpubr)
library(ggsci)
library(flextable)
source("Code/functions.R")
source("Code/summary.R")

metrics_df <- read.csv('metrics_df.csv')
metrics_df_all_n <- read.csv('metrics_df_all_n.csv')
runtime_df <- read.csv('runtime_df.csv') %>% 
  mutate(n_train = stringr::str_count(ses, ",") + 1, .before='ses')

Comparisons

Number of PCs

metrics_df %>% 
  filter(pca == 'bothhands') %>% 
  mutate(npc = as.factor(npc)) %>% 
  group_by(patient, smooth, npc) %>% 
  select(Accuracy, Precision, F1.score, Sensitivity, Specificity) %>% 
  summarize_all(mean) %>% 
  ungroup() %>% 
  pivot_longer(-c(npc, patient, smooth), names_to = "Metric") %>% 
  ggplot(aes(x=Metric, y=value, color=npc)) + 
  geom_boxplot() + 
  facet_wrap(~smooth) + 
  theme_bw() + theme(plot.title = element_text(hjust = 0.5)) +
  scale_color_lancet()

#ggsave('figures/npc_boxplot.png', dpi=300)

Run Time

runtime_df %>% 
  filter(n_train == 1) %>% 
  group_by(npc) %>% 
  summarize(mean_runtime = mean(elapsed))
runtime_df %>% 
  filter(n_train == 1) %>% 
  mutate(npc = as.factor(npc)) %>% 
  ggplot(aes(x=npc, y=elapsed)) + geom_boxplot() + ylab('Run time in seconds') + xlab('Number of PCs') + ggtitle('Run time per number of PCs')

Run times relative to 3 npc.

# relative to 3 npc
runtime_df %>% 
  filter(n_train == 1) %>% 
  group_by(npc) %>% 
  summarize(mean_runtime = mean(elapsed)) %>% 
  mutate(mean_runtime = mean_runtime / min(mean_runtime))

Dominant Hand vs NonDominant Hand vs Both Hands (w. npc=3)

metrics_df_for_plot = metrics_df %>% 
  filter(npc == 3 | npc == 6) %>% 
  mutate(npc = as.factor(npc), patient=as.factor(patient)) %>% 
  group_by(patient, smooth, pca) %>% 
  select(Accuracy, Precision, F1.score, Sensitivity, Specificity) %>% 
  summarize_all(mean) %>% 
  ungroup() 

metrics_df_for_plot %>% 
  pivot_longer(-c(pca, patient, smooth), names_to = "Metric") %>% 
  mutate(pca = recode(.$pca, `bothhands` = "Both Hands", `dominanthand` = "Dominant Hand", `nondominanthand` = "Nondominant Hand")) %>% 
  ggplot(aes(x=Metric, y=value, color=pca)) + 
  geom_boxplot() + 
  facet_wrap(~smooth) + 
  theme_bw() + theme(plot.title = element_text(hjust = 0.5)) +
  scale_color_lancet() + 
  labs(color='PCA')

Testing differences in metrics (paired T-tests)

bothhands = metrics_df_for_plot %>% 
  filter(smooth=='Post-Smoothing', pca=='bothhands') %>% 
  select(where(is.numeric))

domhand = metrics_df_for_plot %>% 
  filter(smooth=='Post-Smoothing', pca=='dominanthand') %>% 
  select(where(is.numeric))

nondomhand = metrics_df_for_plot %>% 
  filter(smooth=='Post-Smoothing', pca=='nondominanthand') %>% 
  select(where(is.numeric))

dom_vs_both = seq_along(bothhands) %>% 
  purrr::map(function(i) t.test(pull(domhand, i), # get i-th column
                                pull(bothhands, i), 
                                paired = TRUE)) %>% 
  purrr::map(broom::tidy) %>% 
  setNames(colnames(bothhands)) %>% 
  dplyr::bind_rows(.id = 'metric') %>% 
  tibble::add_column(comparison = "Dominant hand v. Both Hands", .before = 'metric')
  
nondom_vs_both = seq_along(bothhands) %>% 
  purrr::map(function(i) t.test(pull(nondomhand, i), # get i-th column
                                pull(bothhands, i), 
                                paired = TRUE)) %>% 
  purrr::map(broom::tidy) %>% 
  setNames(colnames(bothhands)) %>% 
  dplyr::bind_rows(.id = 'metric') %>% 
  tibble::add_column(comparison = "Nondominant hand v. Both Hands", .before = 'metric')

nondom_vs_dom = seq_along(bothhands) %>% 
  purrr::map(function(i) t.test(pull(nondomhand, i), # get i-th column
                                pull(domhand, i), 
                                paired = TRUE)) %>% 
  purrr::map(broom::tidy) %>% 
  setNames(colnames(bothhands)) %>% 
  dplyr::bind_rows(.id = 'metric') %>% 
  tibble::add_column(comparison = "Nondominant hand v. Dominant hand", .before = 'metric')

rbind(dom_vs_both, nondom_vs_both, nondom_vs_dom)

Both Hands NPC=3 per hand

Training 1 med session

metrics_df_long = metrics_df %>% filter(pca == 'bothhands', npc==3) %>% group_by(patient, smooth) %>% select(Accuracy, Precision, F1.score, Sensitivity, Specificity) %>% summarize_all(mean) %>% pivot_longer(-c(patient, smooth), names_to = "Metric")

metrics_df_long %>% ggplot(aes(x=Metric, y = value, color=Metric)) + geom_jitter() + geom_boxplot(alpha=0) + facet_wrap(~smooth) + 
  theme_bw() + ggtitle("Summary metrics for Movelet prediction (1 session training)") + theme(plot.title = element_text(hjust = 0.5)) +
  scale_color_d3()

Metrics across sessions

# Prepare table with smoothing
table1 = metrics_df %>% filter(smooth=="Pre-Smoothing", pca == 'bothhands', npc==3) %>% group_by(session) %>% select(Accuracy, Precision, F1.score, Sensitivity, Specificity) %>% summarize_all(mean)

# Prepare table without smoothing
table2 = metrics_df %>% filter(smooth=="Post-Smoothing", pca == 'bothhands', npc==3) %>% group_by(session) %>% select(Accuracy, Precision, F1.score, Sensitivity, Specificity) %>% summarize_all(mean) %>% as.data.frame()

# Join tables
tables = cbind(table1, table2[, -1]) %>% round(2)
colnames(tables)[2:11] = paste0(colnames(tables)[2:11], rep(c(".T", ".F"), each = 5))
headers = data.frame(col_keys = colnames(tables), 
                     from = c("Session", rep(c("Pre-Smoothing", "Post-Smoothing"), each=5)),
                     names = c("Session", colnames(cbind(table1[-1], table2[, -1])))
                     )
# Print table as flextable
tables %>% flextable::flextable() %>% 
  set_header_df(mapping = headers, key = "col_keys") %>% 
  merge_h(part="header") %>% 
  merge_v(part="header") %>% 
  theme_vanilla() %>% 
  width(width = 27, unit = "in") %>% 
  align(align="center", part="all") %>% 
  vline(part="header")

Metrics across training modes

metrics_df_all_n %>% 
  filter(pca == 'bothhands', npc==3) %>% 
  group_by(patient, n_train, smooth) %>% 
  select(Accuracy, Precision, F1.score, Sensitivity, Specificity) %>% 
  summarize_all(mean) %>% 
  pivot_longer(-c(patient, n_train, smooth),  names_to = "Metric") %>% 
  ggplot(aes(x=n_train, y = value, color=as.factor(n_train))) + 
  geom_jitter() + facet_grid(smooth~Metric) +  geom_boxplot(alpha=0) + 
  theme_bw() + ggtitle("Summary metrics for Movelet prediction (n session training)") + 
  theme(plot.title = element_text(hjust = 0.5)) + guides(color=guide_legend(title="N trained med sessions")) + xlab("Train N") +
  scale_color_lancet() 

Both Hands NPC=4

Training 1 med session

metrics_df_long = metrics_df %>% filter(pca == 'bothhands', npc==4) %>% group_by(patient, smooth) %>% select(Accuracy, Precision, F1.score, Sensitivity, Specificity) %>% summarize_all(mean) %>% pivot_longer(-c(patient, smooth), names_to = "Metric")

metrics_df_long %>% ggplot(aes(x=Metric, y = value, color=Metric)) + geom_jitter() + geom_boxplot(alpha=0) + facet_wrap(~smooth) + 
  theme_bw() + ggtitle("Summary metrics for Movelet prediction (1 session training)") + theme(plot.title = element_text(hjust = 0.5)) +
  scale_color_d3()

Metrics across sessions

# Prepare table with smoothing
table1 = metrics_df %>% filter(smooth=="Pre-Smoothing", pca == 'bothhands', npc==4) %>% group_by(session) %>% select(Accuracy, Precision, F1.score, Sensitivity, Specificity) %>% summarize_all(mean)

# Prepare table without smoothing
table2 = metrics_df %>% filter(smooth=="Post-Smoothing", pca == 'bothhands', npc==4) %>% group_by(session) %>% select(Accuracy, Precision, F1.score, Sensitivity, Specificity) %>% summarize_all(mean) %>% as.data.frame()

# Join tables
tables = cbind(table1, table2[, -1]) %>% round(2)
colnames(tables)[2:11] = paste0(colnames(tables)[2:11], rep(c(".T", ".F"), each = 5))
headers = data.frame(col_keys = colnames(tables), 
                     from = c("Session", rep(c("Pre-Smoothing", "Post-Smoothing"), each=5)),
                     names = c("Session", colnames(cbind(table1[-1], table2[, -1])))
                     )
# Print table as flextable
tables %>% flextable::flextable() %>% 
  set_header_df(mapping = headers, key = "col_keys") %>% 
  merge_h(part="header") %>% 
  merge_v(part="header") %>% 
  theme_vanilla() %>% 
  width(width = 27, unit = "in") %>% 
  align(align="center", part="all") %>% 
  vline(part="header")

Metrics across training modes

metrics_df_all_n %>% 
  filter(pca == 'bothhands', npc==4) %>% 
  group_by(patient, n_train, smooth) %>% 
  select(Accuracy, Precision, F1.score, Sensitivity, Specificity) %>% 
  summarize_all(mean) %>% 
  pivot_longer(-c(patient, n_train, smooth),  names_to = "Metric") %>% 
  ggplot(aes(x=n_train, y = value, color=as.factor(n_train))) + 
  geom_jitter() + facet_grid(smooth~Metric) +  geom_boxplot(alpha=0) + 
  theme_bw() + ggtitle("Summary metrics for Movelet prediction (n session training)") + 
  theme(plot.title = element_text(hjust = 0.5)) + guides(color=guide_legend(title="N trained med sessions")) + xlab("Train N") +
  scale_color_lancet() 

Both Hands NPC=5

Training 1 med session

metrics_df_long = metrics_df %>% filter(pca == 'bothhands', npc==5) %>% group_by(patient, smooth) %>% select(Accuracy, Precision, F1.score, Sensitivity, Specificity) %>% summarize_all(mean) %>% pivot_longer(-c(patient, smooth), names_to = "Metric")

metrics_df_long %>% ggplot(aes(x=Metric, y = value, color=Metric)) + geom_jitter() + geom_boxplot(alpha=0) + facet_wrap(~smooth) + 
  theme_bw() + ggtitle("Summary metrics for Movelet prediction (1 session training)") + theme(plot.title = element_text(hjust = 0.5)) +
  scale_color_d3()

Metrics across sessions

# Prepare table with smoothing
table1 = metrics_df %>% filter(smooth=="Pre-Smoothing", pca == 'bothhands', npc==5) %>% group_by(session) %>% select(Accuracy, Precision, F1.score, Sensitivity, Specificity) %>% summarize_all(mean)

# Prepare table without smoothing
table2 = metrics_df %>% filter(smooth=="Post-Smoothing", pca == 'bothhands', npc==5) %>% group_by(session) %>% select(Accuracy, Precision, F1.score, Sensitivity, Specificity) %>% summarize_all(mean) %>% as.data.frame()

# Join tables
tables = cbind(table1, table2[, -1]) %>% round(2)
colnames(tables)[2:11] = paste0(colnames(tables)[2:11], rep(c(".T", ".F"), each = 5))
headers = data.frame(col_keys = colnames(tables), 
                     from = c("Session", rep(c("Pre-Smoothing", "Post-Smoothing"), each=5)),
                     names = c("Session", colnames(cbind(table1[-1], table2[, -1])))
                     )
# Print table as flextable
tables %>% flextable::flextable() %>% 
  set_header_df(mapping = headers, key = "col_keys") %>% 
  merge_h(part="header") %>% 
  merge_v(part="header") %>% 
  theme_vanilla() %>% 
  width(width = 27, unit = "in") %>% 
  align(align="center", part="all") %>% 
  vline(part="header")

Metrics across training modes

metrics_df_all_n %>% 
  filter(pca == 'bothhands', npc==5) %>% 
  group_by(patient, n_train, smooth) %>% 
  select(Accuracy, Precision, F1.score, Sensitivity, Specificity) %>% 
  summarize_all(mean) %>% 
  pivot_longer(-c(patient, n_train, smooth),  names_to = "Metric") %>% 
  ggplot(aes(x=n_train, y = value, color=as.factor(n_train))) + 
  geom_jitter() + facet_grid(smooth~Metric) +  geom_boxplot(alpha=0) + 
  theme_bw() + ggtitle("Summary metrics for Movelet prediction (n session training)") + 
  theme(plot.title = element_text(hjust = 0.5)) + guides(color=guide_legend(title="N trained med sessions")) + xlab("Train N") +
  scale_color_lancet() 

Both Hands NPC=6

Training 1 med session

metrics_df_long = metrics_df %>% filter(pca == 'bothhands', npc==6) %>% group_by(patient, smooth) %>% select(Accuracy, Precision, F1.score, Sensitivity, Specificity) %>% summarize_all(mean) %>% pivot_longer(-c(patient, smooth), names_to = "Metric")

metrics_df_long %>% ggplot(aes(x=Metric, y = value, color=Metric)) + geom_jitter() + geom_boxplot(alpha=0) + facet_wrap(~smooth) + 
  theme_bw() + ggtitle("Summary metrics for Movelet prediction (1 session training)") + theme(plot.title = element_text(hjust = 0.5)) +
  scale_color_d3()

Metrics across sessions

# Prepare table with smoothing
table1 = metrics_df %>% filter(smooth=="Pre-Smoothing", pca == 'bothhands', npc==6) %>% group_by(session) %>% select(Accuracy, Precision, F1.score, Sensitivity, Specificity) %>% summarize_all(mean)

# Prepare table without smoothing
table2 = metrics_df %>% filter(smooth=="Post-Smoothing", pca == 'bothhands', npc==6) %>% group_by(session) %>% select(Accuracy, Precision, F1.score, Sensitivity, Specificity) %>% summarize_all(mean) %>% as.data.frame()

# Join tables
tables = cbind(table1, table2[, -1]) %>% round(2)
colnames(tables)[2:11] = paste0(colnames(tables)[2:11], rep(c(".T", ".F"), each = 5))
headers = data.frame(col_keys = colnames(tables), 
                     from = c("Session", rep(c("Pre-Smoothing", "Post-Smoothing"), each=5)),
                     names = c("Session", colnames(cbind(table1[-1], table2[, -1])))
                     )
# Print table as flextable
tables %>% flextable::flextable() %>% 
  set_header_df(mapping = headers, key = "col_keys") %>% 
  merge_h(part="header") %>% 
  merge_v(part="header") %>% 
  theme_vanilla() %>% 
  width(width = 27, unit = "in") %>% 
  align(align="center", part="all") %>% 
  vline(part="header")

Metrics across training modes

metrics_df_all_n %>% 
  filter(pca == 'bothhands', npc==6) %>% 
  group_by(patient, n_train, smooth) %>% 
  select(Accuracy, Precision, F1.score, Sensitivity, Specificity) %>% 
  summarize_all(mean) %>% 
  pivot_longer(-c(patient, n_train, smooth),  names_to = "Metric") %>% 
  ggplot(aes(x=n_train, y = value, color=as.factor(n_train))) + 
  geom_jitter() + facet_grid(smooth~Metric) +  geom_boxplot(alpha=0) + 
  theme_bw() + ggtitle("Summary metrics for Movelet prediction (n session training)") + 
  theme(plot.title = element_text(hjust = 0.5)) + guides(color=guide_legend(title="N trained med sessions")) + xlab("Train N") +
  scale_color_lancet() 

Both Hands NPC=7

Training 1 med session

metrics_df_long = metrics_df %>% filter(pca == 'bothhands', npc==7) %>% group_by(patient, smooth) %>% select(Accuracy, Precision, F1.score, Sensitivity, Specificity) %>% summarize_all(mean) %>% pivot_longer(-c(patient, smooth), names_to = "Metric")

metrics_df_long %>% ggplot(aes(x=Metric, y = value, color=Metric)) + geom_jitter() + geom_boxplot(alpha=0) + facet_wrap(~smooth) + 
  theme_bw() + ggtitle("Summary metrics for Movelet prediction (1 session training)") + theme(plot.title = element_text(hjust = 0.5)) +
  scale_color_d3()

Metrics across sessions

# Prepare table with smoothing
table1 = metrics_df %>% filter(smooth=="Pre-Smoothing", pca == 'bothhands', npc==7) %>% group_by(session) %>% select(Accuracy, Precision, F1.score, Sensitivity, Specificity) %>% summarize_all(mean)

# Prepare table without smoothing
table2 = metrics_df %>% filter(smooth=="Post-Smoothing", pca == 'bothhands', npc==7) %>% group_by(session) %>% select(Accuracy, Precision, F1.score, Sensitivity, Specificity) %>% summarize_all(mean) %>% as.data.frame()

# Join tables
tables = cbind(table1, table2[, -1]) %>% round(2)
colnames(tables)[2:11] = paste0(colnames(tables)[2:11], rep(c(".T", ".F"), each = 5))
headers = data.frame(col_keys = colnames(tables), 
                     from = c("Session", rep(c("Pre-Smoothing", "Post-Smoothing"), each=5)),
                     names = c("Session", colnames(cbind(table1[-1], table2[, -1])))
                     )
# Print table as flextable
tables %>% flextable::flextable() %>% 
  set_header_df(mapping = headers, key = "col_keys") %>% 
  merge_h(part="header") %>% 
  merge_v(part="header") %>% 
  theme_vanilla() %>% 
  width(width = 27, unit = "in") %>% 
  align(align="center", part="all") %>% 
  vline(part="header")

Metrics across training modes

metrics_df_all_n %>% 
  filter(pca == 'bothhands', npc==7) %>% 
  group_by(patient, n_train, smooth) %>% 
  select(Accuracy, Precision, F1.score, Sensitivity, Specificity) %>% 
  summarize_all(mean) %>% 
  pivot_longer(-c(patient, n_train, smooth),  names_to = "Metric") %>% 
  ggplot(aes(x=n_train, y = value, color=as.factor(n_train))) + 
  geom_jitter() + facet_grid(smooth~Metric) +  geom_boxplot(alpha=0) + 
  theme_bw() + ggtitle("Summary metrics for Movelet prediction (n session training)") + 
  theme(plot.title = element_text(hjust = 0.5)) + guides(color=guide_legend(title="N trained med sessions")) + xlab("Train N") +
  scale_color_lancet() 

Both Hands NPC=8

Training 1 med session

metrics_df_long = metrics_df %>% filter(pca == 'bothhands', npc==8) %>% group_by(patient, smooth) %>% select(Accuracy, Precision, F1.score, Sensitivity, Specificity) %>% summarize_all(mean) %>% pivot_longer(-c(patient, smooth), names_to = "Metric")

metrics_df_long %>% ggplot(aes(x=Metric, y = value, color=Metric)) + geom_jitter() + geom_boxplot(alpha=0) + facet_wrap(~smooth) + 
  theme_bw() + ggtitle("Summary metrics for Movelet prediction (1 session training)") + theme(plot.title = element_text(hjust = 0.5)) +
  scale_color_d3()

Metrics across sessions

# Prepare table with smoothing
table1 = metrics_df %>% filter(smooth=="Pre-Smoothing", pca == 'bothhands', npc==8) %>% group_by(session) %>% select(Accuracy, Precision, F1.score, Sensitivity, Specificity) %>% summarize_all(mean)

# Prepare table without smoothing
table2 = metrics_df %>% filter(smooth=="Post-Smoothing", pca == 'bothhands', npc==8) %>% group_by(session) %>% select(Accuracy, Precision, F1.score, Sensitivity, Specificity) %>% summarize_all(mean) %>% as.data.frame()

# Join tables
tables = cbind(table1, table2[, -1]) %>% round(2)
colnames(tables)[2:11] = paste0(colnames(tables)[2:11], rep(c(".T", ".F"), each = 5))
headers = data.frame(col_keys = colnames(tables), 
                     from = c("Session", rep(c("Pre-Smoothing", "Post-Smoothing"), each=5)),
                     names = c("Session", colnames(cbind(table1[-1], table2[, -1])))
                     )
# Print table as flextable
tables %>% flextable::flextable() %>% 
  set_header_df(mapping = headers, key = "col_keys") %>% 
  merge_h(part="header") %>% 
  merge_v(part="header") %>% 
  theme_vanilla() %>% 
  width(width = 27, unit = "in") %>% 
  align(align="center", part="all") %>% 
  vline(part="header")

Metrics across training modes

metrics_df_all_n %>% 
  filter(pca == 'bothhands', npc==8) %>% 
  group_by(patient, n_train, smooth) %>% 
  select(Accuracy, Precision, F1.score, Sensitivity, Specificity) %>% 
  summarize_all(mean) %>% 
  pivot_longer(-c(patient, n_train, smooth),  names_to = "Metric") %>% 
  ggplot(aes(x=n_train, y = value, color=as.factor(n_train))) + 
  geom_jitter() + facet_grid(smooth~Metric) +  geom_boxplot(alpha=0) + 
  theme_bw() + ggtitle("Summary metrics for Movelet prediction (n session training)") + 
  theme(plot.title = element_text(hjust = 0.5)) + guides(color=guide_legend(title="N trained med sessions")) + xlab("Train N") +
  scale_color_lancet() 

Both Hands NPC=9

Training 1 med session

metrics_df_long = metrics_df %>% filter(pca == 'bothhands', npc==9) %>% group_by(patient, smooth) %>% select(Accuracy, Precision, F1.score, Sensitivity, Specificity) %>% summarize_all(mean) %>% pivot_longer(-c(patient, smooth), names_to = "Metric")

metrics_df_long %>% ggplot(aes(x=Metric, y = value, color=Metric)) + geom_jitter() + geom_boxplot(alpha=0) + facet_wrap(~smooth) + 
  theme_bw() + ggtitle("Summary metrics for Movelet prediction (1 session training)") + theme(plot.title = element_text(hjust = 0.5)) +
  scale_color_d3()

Metrics across sessions

# Prepare table with smoothing
table1 = metrics_df %>% filter(smooth=="Pre-Smoothing", pca == 'bothhands', npc==9) %>% group_by(session) %>% select(Accuracy, Precision, F1.score, Sensitivity, Specificity) %>% summarize_all(mean)

# Prepare table without smoothing
table2 = metrics_df %>% filter(smooth=="Post-Smoothing", pca == 'bothhands', npc==9) %>% group_by(session) %>% select(Accuracy, Precision, F1.score, Sensitivity, Specificity) %>% summarize_all(mean) %>% as.data.frame()

# Join tables
tables = cbind(table1, table2[, -1]) %>% round(2)
colnames(tables)[2:11] = paste0(colnames(tables)[2:11], rep(c(".T", ".F"), each = 5))
headers = data.frame(col_keys = colnames(tables), 
                     from = c("Session", rep(c("Pre-Smoothing", "Post-Smoothing"), each=5)),
                     names = c("Session", colnames(cbind(table1[-1], table2[, -1])))
                     )
# Print table as flextable
tables %>% flextable::flextable() %>% 
  set_header_df(mapping = headers, key = "col_keys") %>% 
  merge_h(part="header") %>% 
  merge_v(part="header") %>% 
  theme_vanilla() %>% 
  width(width = 27, unit = "in") %>% 
  align(align="center", part="all") %>% 
  vline(part="header")

Metrics across training modes

metrics_df_all_n %>% 
  filter(pca == 'bothhands', npc==9) %>% 
  group_by(patient, n_train, smooth) %>% 
  select(Accuracy, Precision, F1.score, Sensitivity, Specificity) %>% 
  summarize_all(mean) %>% 
  pivot_longer(-c(patient, n_train, smooth),  names_to = "Metric") %>% 
  ggplot(aes(x=n_train, y = value, color=as.factor(n_train))) + 
  geom_jitter() + facet_grid(smooth~Metric) +  geom_boxplot(alpha=0) + 
  theme_bw() + ggtitle("Summary metrics for Movelet prediction (n session training)") + 
  theme(plot.title = element_text(hjust = 0.5)) + guides(color=guide_legend(title="N trained med sessions")) + xlab("Train N") +
  scale_color_lancet() 

Both Hands NPC=10

Training 1 med session

metrics_df_long = metrics_df %>% filter(pca == 'bothhands', npc==10) %>% group_by(patient, smooth) %>% select(Accuracy, Precision, F1.score, Sensitivity, Specificity) %>% summarize_all(mean) %>% pivot_longer(-c(patient, smooth), names_to = "Metric")

metrics_df_long %>% ggplot(aes(x=Metric, y = value, color=Metric)) + geom_jitter() + geom_boxplot(alpha=0) + facet_wrap(~smooth) + 
  theme_bw() + ggtitle("Summary metrics for Movelet prediction (1 session training)") + theme(plot.title = element_text(hjust = 0.5)) +
  scale_color_d3()

Metrics across sessions

# Prepare table with smoothing
table1 = metrics_df %>% filter(smooth=="Pre-Smoothing", pca == 'bothhands', npc==10) %>% group_by(session) %>% select(Accuracy, Precision, F1.score, Sensitivity, Specificity) %>% summarize_all(mean)

# Prepare table without smoothing
table2 = metrics_df %>% filter(smooth=="Post-Smoothing", pca == 'bothhands', npc==10) %>% group_by(session) %>% select(Accuracy, Precision, F1.score, Sensitivity, Specificity) %>% summarize_all(mean) %>% as.data.frame()

# Join tables
tables = cbind(table1, table2[, -1]) %>% round(2)
colnames(tables)[2:11] = paste0(colnames(tables)[2:11], rep(c(".T", ".F"), each = 5))
headers = data.frame(col_keys = colnames(tables), 
                     from = c("Session", rep(c("Pre-Smoothing", "Post-Smoothing"), each=5)),
                     names = c("Session", colnames(cbind(table1[-1], table2[, -1])))
                     )
# Print table as flextable
tables %>% flextable::flextable() %>% 
  set_header_df(mapping = headers, key = "col_keys") %>% 
  merge_h(part="header") %>% 
  merge_v(part="header") %>% 
  theme_vanilla() %>% 
  width(width = 27, unit = "in") %>% 
  align(align="center", part="all") %>% 
  vline(part="header")

Metrics across training modes

metrics_df_all_n %>% 
  filter(pca == 'bothhands', npc==10) %>% 
  group_by(patient, n_train, smooth) %>% 
  select(Accuracy, Precision, F1.score, Sensitivity, Specificity) %>% 
  summarize_all(mean) %>% 
  pivot_longer(-c(patient, n_train, smooth),  names_to = "Metric") %>% 
  ggplot(aes(x=n_train, y = value, color=as.factor(n_train))) + 
  geom_jitter() + facet_grid(smooth~Metric) +  geom_boxplot(alpha=0) + 
  theme_bw() + ggtitle("Summary metrics for Movelet prediction (n session training)") + 
  theme(plot.title = element_text(hjust = 0.5)) + guides(color=guide_legend(title="N trained med sessions")) + xlab("Train N") +
  scale_color_lancet() 

Dominant Hand

Training 1 med session

metrics_df_long = metrics_df %>% filter(pca == 'dominanthand') %>% group_by(patient, smooth) %>% select(Accuracy, Precision, F1.score, Sensitivity, Specificity) %>% summarize_all(mean) %>% pivot_longer(-c(patient, smooth), names_to = "Metric")

metrics_df_long %>% ggplot(aes(x=Metric, y = value, color=Metric)) + geom_jitter() + geom_boxplot(alpha=0) + facet_wrap(~smooth) + 
  theme_bw() + ggtitle("Summary metrics for Movelet prediction (1 session training)") + theme(plot.title = element_text(hjust = 0.5)) +
  scale_color_d3()

Metrics across sessions

# Prepare table with smoothing
table1 = metrics_df %>% filter(smooth=="Pre-Smoothing", pca == 'dominanthand') %>% group_by(session) %>% select(Accuracy, Precision, F1.score, Sensitivity, Specificity) %>% summarize_all(mean)

# Prepare table without smoothing
table2 = metrics_df %>% filter(smooth=="Post-Smoothing", pca == 'dominanthand') %>% group_by(session) %>% select(Accuracy, Precision, F1.score, Sensitivity, Specificity) %>% summarize_all(mean) %>% as.data.frame()

# Join tables
tables = cbind(table1, table2[, -1]) %>% round(2)
colnames(tables)[2:11] = paste0(colnames(tables)[2:11], rep(c(".T", ".F"), each = 5))
headers = data.frame(col_keys = colnames(tables), 
                     from = c("Session", rep(c("Pre-Smoothing", "Post-Smoothing"), each=5)),
                     names = c("Session", colnames(cbind(table1[-1], table2[, -1])))
                     )
# Print table as flextable
tables %>% flextable::flextable() %>% 
  set_header_df(mapping = headers, key = "col_keys") %>% 
  merge_h(part="header") %>% 
  merge_v(part="header") %>% 
  theme_vanilla() %>% 
  width(width = 27, unit = "in") %>% 
  align(align="center", part="all") %>% 
  vline(part="header")

Metrics across training modes

metrics_df_all_n %>% 
  filter(pca == 'dominanthand') %>% 
  group_by(patient, n_train, smooth) %>% 
  select(Accuracy, Precision, F1.score, Sensitivity, Specificity) %>% 
  summarize_all(mean) %>% 
  pivot_longer(-c(patient, n_train, smooth),  names_to = "Metric") %>% 
  ggplot(aes(x=n_train, y = value, color=as.factor(n_train))) + 
  geom_jitter() + facet_grid(smooth~Metric) +  geom_boxplot(alpha=0) + 
  theme_bw() + ggtitle("Summary metrics for Movelet prediction (n session training)") + 
  theme(plot.title = element_text(hjust = 0.5)) + guides(color=guide_legend(title="N trained med sessions")) + xlab("Train N") +
  scale_color_lancet() 

NonDominant Hand

Training 1 med session

metrics_df_long = metrics_df %>% filter(pca == 'nondominanthand') %>% group_by(patient, smooth) %>% select(Accuracy, Precision, F1.score, Sensitivity, Specificity) %>% summarize_all(mean) %>% pivot_longer(-c(patient, smooth), names_to = "Metric")

metrics_df_long %>% ggplot(aes(x=Metric, y = value, color=Metric)) + geom_jitter() + geom_boxplot(alpha=0) + facet_wrap(~smooth) + 
  theme_bw() + ggtitle("Summary metrics for Movelet prediction (1 session training)") + theme(plot.title = element_text(hjust = 0.5)) +
  scale_color_d3()

Metrics across sessions

# Prepare table with smoothing
table1 = metrics_df %>% filter(smooth=="Pre-Smoothing", pca == 'nondominanthand') %>% group_by(session) %>% select(Accuracy, Precision, F1.score, Sensitivity, Specificity) %>% summarize_all(mean)

# Prepare table without smoothing
table2 = metrics_df %>% filter(smooth=="Post-Smoothing", pca == 'nondominanthand') %>% group_by(session) %>% select(Accuracy, Precision, F1.score, Sensitivity, Specificity) %>% summarize_all(mean) %>% as.data.frame()

# Join tables
tables = cbind(table1, table2[, -1]) %>% round(2)
colnames(tables)[2:11] = paste0(colnames(tables)[2:11], rep(c(".T", ".F"), each = 5))
headers = data.frame(col_keys = colnames(tables), 
                     from = c("Session", rep(c("Pre-Smoothing", "Post-Smoothing"), each=5)),
                     names = c("Session", colnames(cbind(table1[-1], table2[, -1])))
                     )
# Print table as flextable
tables %>% flextable::flextable() %>% 
  set_header_df(mapping = headers, key = "col_keys") %>% 
  merge_h(part="header") %>% 
  merge_v(part="header") %>% 
  theme_vanilla() %>% 
  width(width = 27, unit = "in") %>% 
  align(align="center", part="all") %>% 
  vline(part="header")

Metrics across training modes

metrics_df_all_n %>% 
  filter(pca == 'nondominanthand') %>% 
  group_by(patient, n_train, smooth) %>% 
  select(Accuracy, Precision, F1.score, Sensitivity, Specificity) %>% 
  summarize_all(mean) %>% 
  pivot_longer(-c(patient, n_train, smooth),  names_to = "Metric") %>% 
  ggplot(aes(x=n_train, y = value, color=as.factor(n_train))) + 
  geom_jitter() + facet_grid(smooth~Metric) +  geom_boxplot(alpha=0) + 
  theme_bw() + ggtitle("Summary metrics for Movelet prediction (n session training)") + 
  theme(plot.title = element_text(hjust = 0.5)) + guides(color=guide_legend(title="N trained med sessions")) + xlab("Train N") +
  scale_color_lancet()